{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## 5.4 处理缺失数据 Handling Missing Data\r\n",
    "\r\n",
    "+ `.dropna(axis=, how=\"any\", thresh=)`：根据各标签的值中是否存在缺失数据对轴标签进行过滤，可以通过阈值调节对缺失值的容忍度；\r\n",
    "+ `.fillna(value, method=, axis=, inplace=, limit=)`：用指定值或插值方法（如ffill、bfill）填充缺失数据\r\n",
    "+ `.isnull()`：返回布尔数组，表示哪些值是缺失的；\r\n",
    "+ `.notnull()`：返回布尔数组，表示哪些值不是缺失的；"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 模块导入\r\n",
    "import os, sys\r\n",
    "sys.path.append(os.path.dirname(os.getcwd()))\r\n",
    "import numpy\r\n",
    "from numpy import NaN\r\n",
    "import pandas\r\n",
    "from dependency import arr_info"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 绪  论"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "ser0_1 = pandas.Series([2, 4, 3, NaN, 1, 5, None])\r\n",
    "\r\n",
    "# Python的None类型，被转换为numpy的NaN，并且isnull()方法也会被当作NaN来处理\r\n",
    "arr_info([ ser0_1 ])\r\n",
    "arr_info([ ser0_1.isnull() ])"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 5.4.1 滤除缺失的数据 Filtering Out Missing Data"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# Series\r\n",
    "\r\n",
    "ser1_1 = pandas.Series([1, NaN, 3, 4, NaN, 2, 5])\r\n",
    "arr_info([ ser1_1 ])\r\n",
    "arr_info([ ser1_1.dropna() ])"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 布尔索引可以达到相同的目的\r\n",
    "\r\n",
    "arr_info([ ser1_1[ser1_1.notnull()] ])"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# DataFrame\r\n",
    "\r\n",
    "frame1_1 = pandas.DataFrame([[1, 3, 2, 9], [2, NaN, 4, 6], [NaN, NaN, NaN, NaN], [5, NaN, 8, NaN]])\r\n",
    "\r\n",
    "arr_info([ frame1_1 ])\r\n",
    "arr_info([ frame1_1.dropna() ])                     # 默认丢弃任何含有缺失值的行\r\n",
    "arr_info([ frame1_1.dropna(how=\"all\") ])            # 参数how=\"all\"表示只丢弃全为缺失值的行\r\n",
    "arr_info([ frame1_1.dropna(axis=1, how=\"all\") ])    # axis=1表示对列进行操作"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 宽容度设定：thresh\r\n",
    "\r\n",
    "frame1_2 = pandas.DataFrame(numpy.random.randn(8, 3), columns=[\"a\", \"b\", \"c\"])\r\n",
    "\r\n",
    "# 教材中的.ix()方法已经弃用，请改用.iloc()、.loc()\r\n",
    "frame1_2.iloc[:4, 1] = NaN\r\n",
    "frame1_2.iloc[:2, 2] = NaN\r\n",
    "\r\n",
    "arr_info([ frame1_2 ])\r\n",
    "arr_info([ frame1_2.dropna(thresh=2) ]) # 保留行内至少有2个为非缺失数据的行"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 5.4.2 填充缺失的数据 Filling in Missing Data"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "# 填充缺失数据\r\n",
    "\r\n",
    "frame2_2 = pandas.DataFrame(numpy.random.randn(8, 3), columns=[\"a\", \"b\", \"c\"])\r\n",
    "\r\n",
    "# 教材中的.ix()方法已经弃用，请改用.iloc()、.loc()\r\n",
    "frame2_2.iloc[:4, 1] = NaN\r\n",
    "frame2_2.iloc[:2, 2] = NaN\r\n",
    "\r\n",
    "arr_info([ frame2_2 ])\r\n",
    "arr_info([ frame2_2.fillna(0) ])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235       NaN       NaN\n",
      "1 -0.028682       NaN       NaN\n",
      "2 -0.150852       NaN -0.719055\n",
      "3 -0.588987       NaN -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n",
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235  0.000000  0.000000\n",
      "1 -0.028682  0.000000  0.000000\n",
      "2 -0.150852  0.000000 -0.719055\n",
      "3 -0.588987  0.000000 -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "source": [
    "# 对不同列填充不同的值\r\n",
    "\r\n",
    "arr_info([ frame2_2 ])\r\n",
    "arr_info([ frame2_2.fillna({\"b\": 0, \"c\": 1}) ]) # 传入字典"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235       NaN       NaN\n",
      "1 -0.028682       NaN       NaN\n",
      "2 -0.150852       NaN -0.719055\n",
      "3 -0.588987       NaN -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n",
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235  0.000000  1.000000\n",
      "1 -0.028682  0.000000  1.000000\n",
      "2 -0.150852  0.000000 -0.719055\n",
      "3 -0.588987  0.000000 -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "source": [
    "# 就地修改\r\n",
    "\r\n",
    "# 即修改原始对象（否则默认将修改后的返回一个新对象，原始对象不改变）：inplace\r\n",
    "arr_info([ frame2_2 ])\r\n",
    "frame2_2.fillna(0, inplace=True)\r\n",
    "arr_info([ frame2_2 ])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235       NaN       NaN\n",
      "1 -0.028682       NaN       NaN\n",
      "2 -0.150852       NaN -0.719055\n",
      "3 -0.588987       NaN -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n",
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235  0.000000  0.000000\n",
      "1 -0.028682  0.000000  0.000000\n",
      "2 -0.150852  0.000000 -0.719055\n",
      "3 -0.588987  0.000000 -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "source": [
    "# 插值方法（类似reindex）\r\n",
    "\r\n",
    "# 由于上一个单元格inplace=True进行了修改，导致原始对象被修改，所以此处需要重新赋值\r\n",
    "frame2_2.iloc[:4, 1] = NaN\r\n",
    "frame2_2.iloc[:2, 2] = NaN\r\n",
    "arr_info([ frame2_2 ])\r\n",
    "arr_info([ frame2_2.fillna(method=\"bfill\", limit=2) ])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235  0.000000  0.000000\n",
      "1 -0.028682  0.000000  0.000000\n",
      "2 -0.150852  0.000000 -0.719055\n",
      "3 -0.588987  0.000000 -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n",
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235  0.000000  0.000000\n",
      "1 -0.028682  0.000000  0.000000\n",
      "2 -0.150852  0.000000 -0.719055\n",
      "3 -0.588987  0.000000 -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "source": [
    "# 实用技巧：填充平均数等数据\r\n",
    "\r\n",
    "arr_info([ frame2_2 ])\r\n",
    "frame2_2.fillna(frame2_2.mean())"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "【1】\n",
      " 类 型: <class 'pandas.core.frame.DataFrame'>\n",
      "          a         b         c\n",
      "0 -0.091235  0.000000  0.000000\n",
      "1 -0.028682  0.000000  0.000000\n",
      "2 -0.150852  0.000000 -0.719055\n",
      "3 -0.588987  0.000000 -1.475006\n",
      "4 -0.367583  0.655554 -1.399846\n",
      "5  0.556376 -0.251997  1.043201\n",
      "6  0.454041  0.142228  1.875291\n",
      "7  1.802032  0.769419 -0.762617\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "          a         b         c\n",
       "0 -0.091235  0.000000  0.000000\n",
       "1 -0.028682  0.000000  0.000000\n",
       "2 -0.150852  0.000000 -0.719055\n",
       "3 -0.588987  0.000000 -1.475006\n",
       "4 -0.367583  0.655554 -1.399846\n",
       "5  0.556376 -0.251997  1.043201\n",
       "6  0.454041  0.142228  1.875291\n",
       "7  1.802032  0.769419 -0.762617"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.091235</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.028682</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.150852</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.719055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.588987</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.475006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.367583</td>\n",
       "      <td>0.655554</td>\n",
       "      <td>-1.399846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.556376</td>\n",
       "      <td>-0.251997</td>\n",
       "      <td>1.043201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.454041</td>\n",
       "      <td>0.142228</td>\n",
       "      <td>1.875291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.802032</td>\n",
       "      <td>0.769419</td>\n",
       "      <td>-0.762617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.9.6",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.6 64-bit ('venv': venv)"
  },
  "interpreter": {
   "hash": "104c1c2f6fd29e8a7ac84cead711748fe50d8f2c3e925d91da5793bfa308b93e"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}